{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import category_encoders as ce\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import category_encoders as ce\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.decomposition import truncated_svd\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "num_rows = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting...\n",
      "Merging...\n"
     ]
    }
   ],
   "source": [
    "application_train = pd.read_csv(\"../data/application_train.csv\")\n",
    "application_test = pd.read_csv(\"../data/application_test.csv\")\n",
    "POS_CASH = pd.read_csv('../data/POS_CASH_balance.csv')\n",
    "credit_card = pd.read_csv('../data/credit_card_balance.csv')\n",
    "bureau = pd.read_csv('../data/bureau.csv')\n",
    "previous_app = pd.read_csv('../data/previous_application.csv')\n",
    "subm = pd.read_csv(\"../data/sample_submission.csv\")\n",
    "\n",
    "\n",
    "print(\"Converting...\")\n",
    "le = LabelEncoder()\n",
    "POS_CASH['NAME_CONTRACT_STATUS'] = \\\n",
    "    le.fit_transform(POS_CASH['NAME_CONTRACT_STATUS'].astype(str))\n",
    "nunique_status = \\\n",
    "    POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR')\\\n",
    "        .nunique()[['NAME_CONTRACT_STATUS']]\\\n",
    "        .rename(columns={'NAME_CONTRACT_STATUS': 'NUNIQUE_STATUS_POS_CASH'})\n",
    "nunique_status.reset_index(inplace=True)\n",
    "POS_CASH = POS_CASH.merge(nunique_status, how='left', on='SK_ID_CURR')\n",
    "POS_CASH.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True)\n",
    "\n",
    "credit_card['NAME_CONTRACT_STATUS'] = \\\n",
    "    le.fit_transform(credit_card['NAME_CONTRACT_STATUS'].astype(str))\n",
    "nunique_status = \\\n",
    "    credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR')\\\n",
    "        .nunique()[['NAME_CONTRACT_STATUS']]\\\n",
    "        .rename(columns={'NAME_CONTRACT_STATUS': 'NUNIQUE_STATUS_credit_card'})\n",
    "nunique_status.reset_index(inplace=True)\n",
    "credit_card = credit_card.merge(nunique_status, how='left', on='SK_ID_CURR')\n",
    "credit_card.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True)\n",
    "\n",
    "bureau_cat_features = [f for f in bureau.columns if bureau[f].dtype == 'object']\n",
    "for f in bureau_cat_features:\n",
    "    bureau[f] = le.fit_transform(bureau[f].astype(str))\n",
    "    nunique = bureau[['SK_ID_CURR', f]].groupby('SK_ID_CURR').nunique()[[f]]\\\n",
    "        .rename(columns={f: 'NUNIQUE_'+f+'_bureau'})\n",
    "    nunique.reset_index(inplace=True)\n",
    "    bureau = bureau.merge(nunique, how='left', on='SK_ID_CURR')\n",
    "    bureau.drop([f], axis=1, inplace=True)\n",
    "bureau.drop(['SK_ID_BUREAU'], axis=1, inplace=True)\n",
    "\n",
    "previous_app_cat_features = [f for f in previous_app.columns if previous_app[f].dtype == 'object']\n",
    "for f in previous_app_cat_features:\n",
    "    previous_app[f] = le.fit_transform(previous_app[f].astype(str))\n",
    "    nunique = previous_app[['SK_ID_CURR', f]].groupby('SK_ID_CURR').nunique()[[f]]\\\n",
    "        .rename(columns={f: 'NUNIQUE_'+f+'_previous_app'})\n",
    "    nunique.reset_index(inplace=True)\n",
    "    previous_app = previous_app.merge(nunique, how='left', on='SK_ID_CURR')\n",
    "    previous_app.drop([f], axis=1, inplace=True)\n",
    "previous_app.drop(['SK_ID_PREV'], axis=1, inplace=True)\n",
    "\n",
    "print(\"Merging...\")\n",
    "data_train = application_train.merge(POS_CASH.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                             how='left', on='SK_ID_CURR')\n",
    "data_test = application_test.merge(POS_CASH.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                           how='left', on='SK_ID_CURR')\n",
    "\n",
    "data_train = data_train.merge(credit_card.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                         how='left', on='SK_ID_CURR')\n",
    "data_test = data_test.merge(credit_card.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                       how='left', on='SK_ID_CURR')\n",
    "                                                       \n",
    "data_train = data_train.merge(bureau.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                    how='left', on='SK_ID_CURR')\n",
    "data_test = data_test.merge(bureau.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                  how='left', on='SK_ID_CURR')\n",
    "                                                  \n",
    "data_train = data_train.merge(previous_app.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                          how='left', on='SK_ID_CURR')\n",
    "data_test = data_test.merge(previous_app.groupby('SK_ID_CURR').mean().reset_index(),\n",
    "                                                        how='left', on='SK_ID_CURR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack/test_catb-0.csv\"\n",
    "validation_file_path = 'Level_1_stack/validation_catb-0.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip', nrows= num_rows)\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features = [f for f in data_train.columns if data_train[f].dtype == 'object']\n",
    "def column_index(df, query_cols):\n",
    "    cols = df.columns.values\n",
    "    sidx = np.argsort(cols)\n",
    "    return sidx[np.searchsorted(cols, query_cols, sorter=sidx)]\n",
    "cat_features_inds = column_index(data_train, cat_features)    \n",
    "print(\"Cat features are: %s\" % [f for f in cat_features])\n",
    "print(cat_features_inds)\n",
    "\n",
    "for col in cat_features:\n",
    "    data_train[col] = le.fit_transform(data_train[col].astype(str))\n",
    "    data_test[col] = le.fit_transform(data_test[col].astype(str))\n",
    "    \n",
    "data_train.fillna(-1, inplace=True)\n",
    "data_test.fillna(-1, inplace=True)\n",
    "cols = data_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = data_train\n",
    "test_df = data_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting LightGBM. Train shape: (307511, 199), test shape: (307511, 199)\n",
      "(307511, 197)\n",
      "(246008, 197) (61503, 197) (307511, 197)\n",
      "\n",
      "CatBoost...\n",
      "Warning: Parameter 'use_best_model' is true, thus evaluation metric iscalculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
      "0:\ttest: 0.7084029\tbest: 0.7084029 (0)\ttotal: 245ms\tremaining: 4m 4s\n",
      "50:\ttest: 0.7678743\tbest: 0.7678743 (50)\ttotal: 12.8s\tremaining: 3m 58s\n",
      "100:\ttest: 0.7750154\tbest: 0.7750154 (100)\ttotal: 25.4s\tremaining: 3m 46s\n",
      "150:\ttest: 0.7782168\tbest: 0.7782168 (150)\ttotal: 38s\tremaining: 3m 33s\n",
      "200:\ttest: 0.7797442\tbest: 0.7797442 (200)\ttotal: 50.6s\tremaining: 3m 21s\n",
      "250:\ttest: 0.7803331\tbest: 0.7803331 (250)\ttotal: 1m 3s\tremaining: 3m 8s\n",
      "300:\ttest: 0.7810142\tbest: 0.7810142 (300)\ttotal: 1m 15s\tremaining: 2m 56s\n",
      "350:\ttest: 0.7812221\tbest: 0.7812221 (350)\ttotal: 1m 28s\tremaining: 2m 43s\n",
      "Stopped by overfitting detector  (45 iterations wait)\n",
      "\n",
      "bestTest = 0.7812220604\n",
      "bestIteration = 350\n",
      "\n",
      "Shrink model to first 351 iterations.\n",
      "Fold  1 AUC : 0.781222\n",
      "(246009, 197) (61502, 197) (307511, 197)\n",
      "\n",
      "CatBoost...\n",
      "Warning: Parameter 'use_best_model' is true, thus evaluation metric iscalculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
      "0:\ttest: 0.7083629\tbest: 0.7083629 (0)\ttotal: 248ms\tremaining: 4m 7s\n",
      "50:\ttest: 0.7668041\tbest: 0.7668041 (50)\ttotal: 12.7s\tremaining: 3m 56s\n",
      "100:\ttest: 0.7744288\tbest: 0.7744288 (100)\ttotal: 25.4s\tremaining: 3m 45s\n",
      "150:\ttest: 0.7774287\tbest: 0.7774287 (150)\ttotal: 38s\tremaining: 3m 33s\n",
      "200:\ttest: 0.7794338\tbest: 0.7794338 (200)\ttotal: 50.6s\tremaining: 3m 21s\n",
      "250:\ttest: 0.7803883\tbest: 0.7803883 (250)\ttotal: 1m 3s\tremaining: 3m 8s\n",
      "300:\ttest: 0.7806929\tbest: 0.7806929 (300)\ttotal: 1m 15s\tremaining: 2m 56s\n",
      "350:\ttest: 0.7811776\tbest: 0.7811776 (350)\ttotal: 1m 28s\tremaining: 2m 43s\n",
      "400:\ttest: 0.7811895\tbest: 0.7811895 (400)\ttotal: 1m 41s\tremaining: 2m 31s\n",
      "Stopped by overfitting detector  (45 iterations wait)\n",
      "\n",
      "bestTest = 0.781189506\n",
      "bestIteration = 400\n",
      "\n",
      "Shrink model to first 401 iterations.\n",
      "Fold  2 AUC : 0.781190\n",
      "(246009, 197) (61502, 197) (307511, 197)\n",
      "\n",
      "CatBoost...\n",
      "Warning: Parameter 'use_best_model' is true, thus evaluation metric iscalculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
      "0:\ttest: 0.7023835\tbest: 0.7023835 (0)\ttotal: 243ms\tremaining: 4m 2s\n",
      "50:\ttest: 0.7605390\tbest: 0.7605390 (50)\ttotal: 12.7s\tremaining: 3m 56s\n",
      "100:\ttest: 0.7669236\tbest: 0.7669236 (100)\ttotal: 25.2s\tremaining: 3m 44s\n",
      "150:\ttest: 0.7698498\tbest: 0.7698498 (150)\ttotal: 37.9s\tremaining: 3m 33s\n",
      "200:\ttest: 0.7711399\tbest: 0.7711399 (200)\ttotal: 50.4s\tremaining: 3m 20s\n",
      "250:\ttest: 0.7718008\tbest: 0.7718008 (250)\ttotal: 1m 3s\tremaining: 3m 8s\n",
      "300:\ttest: 0.7721348\tbest: 0.7721348 (300)\ttotal: 1m 15s\tremaining: 2m 55s\n",
      "350:\ttest: 0.7724752\tbest: 0.7724752 (350)\ttotal: 1m 28s\tremaining: 2m 43s\n",
      "Stopped by overfitting detector  (45 iterations wait)\n",
      "\n",
      "bestTest = 0.7724751599\n",
      "bestIteration = 350\n",
      "\n",
      "Shrink model to first 351 iterations.\n",
      "Fold  3 AUC : 0.772475\n",
      "(246009, 197) (61502, 197) (307511, 197)\n",
      "\n",
      "CatBoost...\n",
      "Warning: Parameter 'use_best_model' is true, thus evaluation metric iscalculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
      "0:\ttest: 0.7063006\tbest: 0.7063006 (0)\ttotal: 249ms\tremaining: 4m 8s\n",
      "50:\ttest: 0.7616536\tbest: 0.7616536 (50)\ttotal: 12.8s\tremaining: 3m 57s\n",
      "100:\ttest: 0.7685090\tbest: 0.7685090 (100)\ttotal: 25.5s\tremaining: 3m 46s\n",
      "150:\ttest: 0.7716432\tbest: 0.7716432 (150)\ttotal: 38.1s\tremaining: 3m 33s\n",
      "200:\ttest: 0.7725672\tbest: 0.7725672 (200)\ttotal: 50.7s\tremaining: 3m 21s\n",
      "250:\ttest: 0.7731349\tbest: 0.7731349 (250)\ttotal: 1m 3s\tremaining: 3m 9s\n",
      "300:\ttest: 0.7735468\tbest: 0.7735468 (300)\ttotal: 1m 16s\tremaining: 2m 56s\n",
      "350:\ttest: 0.7737735\tbest: 0.7737735 (350)\ttotal: 1m 28s\tremaining: 2m 44s\n",
      "400:\ttest: 0.7743689\tbest: 0.7743689 (400)\ttotal: 1m 41s\tremaining: 2m 31s\n",
      "450:\ttest: 0.7743741\tbest: 0.7743741 (450)\ttotal: 1m 53s\tremaining: 2m 18s\n",
      "Stopped by overfitting detector  (45 iterations wait)\n",
      "\n",
      "bestTest = 0.7743740772\n",
      "bestIteration = 450\n",
      "\n",
      "Shrink model to first 451 iterations.\n",
      "Fold  4 AUC : 0.774374\n",
      "(246009, 197) (61502, 197) (307511, 197)\n",
      "\n",
      "CatBoost...\n",
      "Warning: Parameter 'use_best_model' is true, thus evaluation metric iscalculated on every iteration. 'metric_period' is ignored for evaluation metric.\n",
      "0:\ttest: 0.7087672\tbest: 0.7087672 (0)\ttotal: 246ms\tremaining: 4m 5s\n",
      "50:\ttest: 0.7672297\tbest: 0.7672297 (50)\ttotal: 12.7s\tremaining: 3m 57s\n",
      "100:\ttest: 0.7742866\tbest: 0.7742866 (100)\ttotal: 25.4s\tremaining: 3m 46s\n",
      "150:\ttest: 0.7767763\tbest: 0.7767763 (150)\ttotal: 38.1s\tremaining: 3m 33s\n",
      "200:\ttest: 0.7783634\tbest: 0.7783634 (200)\ttotal: 50.7s\tremaining: 3m 21s\n",
      "250:\ttest: 0.7792892\tbest: 0.7792892 (250)\ttotal: 1m 3s\tremaining: 3m 9s\n",
      "300:\ttest: 0.7790540\tbest: 0.7792892 (250)\ttotal: 1m 16s\tremaining: 2m 56s\n",
      "Stopped by overfitting detector  (45 iterations wait)\n",
      "\n",
      "bestTest = 0.7792892261\n",
      "bestIteration = 250\n",
      "\n",
      "Shrink model to first 251 iterations.\n",
      "Fold  5 AUC : 0.779289\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding = 'ohe'\n",
    "\n",
    "\n",
    "\n",
    "print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "#feats = [col for col in feats_0 if df[col].dtype == 'object']\n",
    "\n",
    "cat_features = [f for f in train_df[feats].columns if train_df[f].dtype == 'object']\n",
    "\n",
    "print(train_df[feats].shape)\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):\n",
    "        \n",
    "      \n",
    "        categorical_columns = [col for col in train_df.columns if train_df[col].dtype == 'object']\n",
    "        \n",
    "        if encoding == 'ohe':\n",
    "            \n",
    "            enc = ce.OneHotEncoder(impute_missing=True, cols=categorical_columns).fit(train_df[feats].iloc[train_idx],\n",
    "                                                                                       train_df['TARGET'].iloc[train_idx])\n",
    "            x_train = enc.transform(train_df[feats].iloc[train_idx])\n",
    "            x_valid = enc.transform(train_df[feats].iloc[valid_idx])\n",
    "            x_test = enc.transform(test_df[feats])\n",
    "            print(x_train.shape, x_valid.shape, x_test.shape)\n",
    "\n",
    "        print(\"\\nCatBoost...\")                                     \n",
    "        clf = CatBoostClassifier(iterations=1000,\n",
    "                              learning_rate=0.1,\n",
    "                              depth=7,\n",
    "                              l2_leaf_reg=40,\n",
    "                              bootstrap_type='Bernoulli',\n",
    "                              subsample=0.9,\n",
    "                              rsm=0.75,\n",
    "                              scale_pos_weight=5,\n",
    "                              eval_metric='AUC',\n",
    "                              metric_period=50,\n",
    "                              od_type='Iter',\n",
    "                              od_wait=45,\n",
    "                              random_seed=17,\n",
    "                              allow_writing_files=False)\n",
    "\n",
    "        clf.fit(x_train, train_df['TARGET'].iloc[train_idx].values, eval_set=(x_valid, train_df['TARGET'].iloc[valid_idx].values)\n",
    "                , cat_features=[], use_best_model=True, verbose=True)\n",
    "\n",
    "        oof_preds[valid_idx] = clf.predict_proba(x_valid)[:, 1]\n",
    "        sub_preds += clf.predict_proba(x_test)[:, 1] / folds.n_splits\n",
    "\n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(train_df['TARGET'].iloc[valid_idx].values, oof_preds[valid_idx])))\n",
    "        del clf\n",
    "        gc.collect()\n",
    "\n",
    "# print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'].iloc[train_idx].values, oof_preds))\n",
    "# # Write submission file and plot feature importance\n",
    "\n",
    "sub_df = test_df[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "\n",
    "val_df = train_df[['SK_ID_CURR', 'TARGET']].copy()\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)        \n",
    "            \n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
